#!/usr/bin/env python
# -*- coding: utf-8 -*-  
# from optparse import OptionParser
import sys
import os
import re
import time
import datetime
from xml.dom import minidom
# import logging
import logging.config
from pip._vendor.distlib.util import Configurator
#from test.test_normalization import unistr

reload(sys)
sys.setdefaultencoding('utf8')

curdate = datetime.date.today()
ydate = curdate - datetime.timedelta(days=1)
 
print curdate,ydate


xmlfile = 'configLog.xml'
if not os.path.exists(xmlfile):
    print "Conf file %s does not exist" %(xmlfile)

xmldoc = minidom.parse(xmlfile)
configure = xmldoc.getElementsByTagName("configure")[0]
sources = xmldoc.getElementsByTagName("source")


def replaceString(path, oStr, toStr):
    return path.replace(oStr,toStr)
         
def changeTimeFormat(date, hour):
    if re.match(r'\d+:\d+:\d+', hour):
        return str(date) + ' ' + hour
    else:
        return hour

# 定义xml中存放所有infile的列表
fileList = []

for source in sources:
    # 要清洗的文件
    infile = source.attributes["infile"].value
    # 清洗文件的类型 0:普通日志文件  1:DB文件
    iftype = source.attributes["type"].value
    # 日志系统名
    sysname = source.attributes["sysname"].value
    # 清洗后的文件名列表
    fields = source.getElementsByTagName("field")
    
    # 存放清洗后日志文件名(包括所有属性一个字典)的列表
    fieldList = []
    # 定义文件句柄字典
    fileHandle = []
    
    for field in fields:
        fname = field.attributes["name"].value
        regular = field.attributes["regular"].value
        short_reg = field.attributes["short_reg"].value
        outdir = field.attributes["outdir"].value
        delim = field.attributes["delim"].value
        
        t_fname_key = replaceString(fname,'.data','')
        outfile = replaceString(outdir, '#Y-m-d#', str(ydate)) + os.path.sep + sysname + '_' + fname 
        outfile_err = replaceString(outfile,'.data','.err')
        
        fh_file = {
             outfile : t_fname_key,
             outfile_err :  t_fname_key + '_' + 'err'            
        }
        
        fdict = {
            "fname"     : fname,
            "regular"   : regular,
            "short_reg" : short_reg,
            "outdir"    : outdir,
            "delim"     : delim,
        }
        
        fieldList.append(fdict)
        fileHandle.append(fh_file)
        
    # 定义一个带有属性的infile文件字典
    sdict = {
        "infile"    : infile,
        "iftype"    : iftype, 
        "sysname"   : sysname,     
        "fields"    : fieldList,
        "fileHangle": fileHandle               
    }
        
    fileList.append(sdict)
    

for indic in fileList:
    t_infile = replaceString(indic["infile"], '#Y-m-d#', str(ydate))
    t_type = indic["iftype"]
    t_sysname = indic["sysname"]

    unknown_file = "D:\\data\\%s\\%s\\%s_useless.info" %(ydate, t_sysname, t_sysname)
    if not os.path.exists(os.path.dirname(unknown_file)):
        os.makedirs(os.path.dirname(unknown_file))
                
    if not os.path.exists(t_infile):
        print "warnning : %s is not exists" %(t_infile)
        continue
    
    in_handle = open(t_infile, 'rb')
    fh_unknow = open(unknown_file,'w')
    
    tmp_dict = {}
    
    # 打开所有需要写入的文件句柄
    for fh_dict in indic["fileHangle"]:
        for t_outfile,t_fh in fh_dict.items():
            if not os.path.exists(os.path.dirname(t_outfile)):
                os.makedirs(os.path.dirname(t_outfile))
            t_fh = open(t_outfile,'w')
            tmp_dict[t_outfile] = t_fh
    
    print "解析文件 ：'%s' 文件类型：'%s' \n" %(t_infile,t_type)
    # fh_unknow.write("解析文件 ：'%s' 文件类型：'%s' \n" %(t_infile,t_type))
    
    # 清洗后匹配到日志关键词的行数
    count_in = {}
    # 匹配到长正则表达式的行数
    count_out = {}
    
    # 行匹配到正则标志位，0表示未匹配到正则，1表示匹配到正则

    
    for line in in_handle:
        flag = 0
        line = line.strip()
        
        for logname in indic["fields"]:
            t_name = logname["fname"]
            t_regular = logname["regular"]
            t_shot_reg = logname["short_reg"]
            t_outdir = replaceString(logname["outdir"], '#Y-m-d#', str(ydate))
            t_delim = logname["delim"]
            # short_fname = replaceString(t_name,'.data','')
            t_oufile = t_outdir + os.path.sep + t_sysname + '_' + t_name
            t_outfile_err = replaceString(t_oufile,'.data','.err')
        
            # print "    # 日志名 ：'%s'  匹配正则 ：'%s'  简写正则 ：'%s'  输出日志路径 ：'%s'" %(t_name,t_regular,t_shot_reg,t_outdir)
            
            short_match = re.search(r'%s' %(t_shot_reg), line)
    
            # 匹配到正则关键词
            if short_match:
    
                flag = 1
                
                if count_in.has_key(t_name):
                    count_in[t_name] += 1
                else:
                    count_in[t_name] = 1
                # print "++++++++++ %s" %(t_regular)
                
                long_matchs = re.match(t_regular, line)
                
                # 匹配到长关键词
                if long_matchs:
                    if count_out.has_key(t_name):
                        count_out[t_name] += 1
                    else:
                        count_out[t_name] = 1
                        
                    results = list(long_matchs.groups())
                    results[0] = changeTimeFormat(ydate, results[0])
                    #print "%s" %(t_delim)
                    #print "++++%s" %(t_delim)
                    if re.match(r'\\t',t_delim):
                        t_delim = '\t'
                        # print "#### " + t_delim

                    msg = t_delim.join(results)
                    # print msg
                    print "write %s to file : %s" %(msg, t_oufile)
                    tmp_dict[t_oufile].write(msg + '\n')
                # 匹配到正则关键词但是匹配不到长正则
                else:
                    tmp_dict[t_outfile_err].write(line + '\n')
                    print "write %s to file : %s" %(line, t_outfile_err)
                break
            else:
                continue
            
        if flag:
            continue
        else:
            print "write %s to unknown file %s" %(line, unknown_file)
            fh_unknow.write(line + '\n')
            
        
    in_handle.close()
    fh_unknow.close()
    
    # 关闭所有打开的文件句柄
    for t_outfile,t_fh in tmp_dict.items():
        t_fh.close()
        
   
# if __name__ == '__main__':

# last = time.time()



     
